{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'user_agent'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-f6df15e8b2fc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpymysql\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0muser_agent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtomorrow3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mthreads\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'user_agent'"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import string\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import pymysql\n",
    "import requests\n",
    "import user_agent\n",
    "import json\n",
    "from tomorrow3 import threads\n",
    "from pymysql import connect\n",
    "from DBUtils.PooledDB import PooledDB\n",
    "\n",
    "pool = PooledDB(pymysql, 10, host=\"127.0.0.1\", user=\"test\", passwd=\"\", db=\"wenben_book\", port=8889)\n",
    "con = connect(\"localhost\", \"test\", \"\", \"wenben_book\", 8889)\n",
    "\n",
    "\n",
    "def generate_headers():\n",
    "    return {\"user-agent\": user_agent.generate_user_agent()}\n",
    "\n",
    "\n",
    "def generate_cookie():\n",
    "    return {\"bid\": \"\".join(random.sample(string.ascii_letters + string.digits, 11))}\n",
    "\n",
    "\n",
    "def exesql(con: con, sql: str):\n",
    "    cur = con.cursor()\n",
    "    cur.execute(sql)\n",
    "    cur.close()\n",
    "\n",
    "\n",
    "def getproxy():\n",
    "    a = json.loads(requests.get(\"54.190.145.159:8000\").text)\n",
    "    return a\n",
    "\n",
    "\n",
    "\n",
    "@threads(10)\n",
    "def get_bs(url: str, headers: dict, cookie: dict):\n",
    "    bs = BeautifulSoup(requests.get(url, cookies=cookie, headers=headers).text, \"html.parser\")\n",
    "    p = bs.find_all(\"script\", type=\"application/ld+json\")\n",
    "    data = p[0].get_text()\n",
    "\n",
    "    ul = json.loads(data, strict=False)[\"url\"]\n",
    "    exesql(con, f\"insert into test values ('{ul}','{data}')\")\n",
    "\n",
    "\n",
    "for page in range(1, 10000):\n",
    "    print(f\"开始第{page}页\")\n",
    "    url = f\"https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=&start={page * 20}&year_range=2017,2017\"\n",
    "    a = requests.get(url, cookies=generate_cookie(), headers=generate_headers())\n",
    "    print(a.text)\n",
    "    data = json.loads(a.text)\n",
    "    for url in data[\"data\"]:\n",
    "        headers = {\"user-agent\": user_agent.generate_user_agent()}\n",
    "        cookie = {\"bid\": \"\".join(random.sample(string.ascii_letters + string.digits, 11))}\n",
    "        get_bs(url[\"url\"], headers=headers, cookie=cookie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
